from pymongo import MongoClient
import re
client = MongoClient()
client = MongoClient('192.168.0.220', 27017)
db = client['tjdb']
collection = db['Data']


# import requests
import base64
from bs4 import BeautifulSoup
# import assert
def testPage(id):
    mongoData = {}
    url = 'http://www.tjsql.com/data.aspx?d=%s'%str(id)
    try:
        with open('/home/ider/Documents/pages/%s.html'%id,'rt')as f:
            res = f.read()
    except Exception as e:
        print(id)
        return
    # soup = BeautifulSoup(res.text, "html5lib")
    soup = BeautifulSoup(res, "lxml")
    # soup = BeautifulSoup(res.text)

    ##第一段内容
    tables = soup.find('textarea',id='yysy').string
    tables = base64.b64decode(tables).decode('utf8')

    sdemo = soup.find('textarea',id='sdemo').string
    nj = soup.find('input',id='nj')['value']
    stitl = soup.find('input',id='stitl')['value']

    tableName = soup.find('div',class_='einfo').h2.string
    mongoData['id'] = id
    mongoData['表名'] = tableName
    mongoData['数据'] = tables
    mongoData['来源'] = nj
    mongoData['标题'] = stitl
    mongoData['标注'] = sdemo
    # print('表名 : %s'%tableName)
    # print('数据 : %s'%tables)
    # print('来源 : %s'%nj)
    # print('标题 : %s'%stitl)
    # print('标注 : %s'%sdemo)

    ##第三段内容
    elites = []
    ttgls = []
    for elite in soup.find_all('li',class_='elite'):
        id = re.findall('d=(\d*)',elite.a['href'])[0]
        elites.append([id,elite.a.string])
    for ttgl in soup.find_all('span',class_='ttgl'):
        id = re.findall('d=(\d*)',ttgl.a['href'])[0]
        ttgls.append([id,ttgl.a.string])
    mongoData['本栏目数据'] = elites
    mongoData['历年相似数据'] = ttgls
    # print('本栏目数据 : %s'%elites)
    # print('历年相似数据 : %s'%ttgls)

    ###第二段内容
    try:
        for tr in soup.find('table',class_='bbtable').find_all('tr'):
            # name =
            td = tr.find_all('td')
            name = td[0].string.replace('：','').strip()
            if '标签' in name:
                continue
            value = td[1].string.strip()

            # print('%s:%s'%(name,value))

    except AttributeError as e:
        pass
    collection.insert_one(mongoData)



def main():
    for i in range(1,171613):
        try:
            testPage(i)
        except Exception as e:
            print(i)
            # raise
        # break

if __name__=='__main__':
    main()
